// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

/*****************************************************************************/
#ifndef HARDWARE_INTRINSIC
#error Define HARDWARE_INTRINSIC before including this file
#endif
/*****************************************************************************/

// clang-format off

#ifdef FEATURE_HW_INTRINSICS
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
//                 ISA            Function name                                              SIMD size  NumArg                                                                                                                 Instructions                                                                                        Category                           Flags
//                                                                                                                  {TYP_BYTE,           TYP_UBYTE,          TYP_SHORT,          TYP_USHORT,         TYP_INT,            TYP_UINT,           TYP_LONG,           TYP_ULONG,          TYP_FLOAT,          TYP_DOUBLE}
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
//  SVE Intrinsics
#define FIRST_NI_Sve              NI_Sve_Abs
HARDWARE_INTRINSIC(Sve,           Abs,                                                              -1,     -1,     {INS_sve_abs,        INS_invalid,        INS_sve_abs,        INS_invalid,        INS_sve_abs,        INS_invalid,        INS_sve_abs,        INS_invalid,        INS_sve_fabs,       INS_sve_fabs},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           AbsoluteCompareGreaterThan,                                       -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_facgt,      INS_sve_facgt},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           AbsoluteCompareGreaterThanOrEqual,                                -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_facge,      INS_sve_facge},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           AbsoluteCompareLessThan,                                          -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_faclt,      INS_sve_faclt},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           AbsoluteCompareLessThanOrEqual,                                   -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_facle,      INS_sve_facle},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           AbsoluteDifference,                                               -1,     -1,     {INS_sve_sabd,       INS_sve_uabd,       INS_sve_sabd,       INS_sve_uabd,       INS_sve_sabd,       INS_sve_uabd,       INS_sve_sabd,       INS_sve_uabd,       INS_sve_fabd,       INS_sve_fabd},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Add,                                                              -1,     -1,     {INS_sve_add,        INS_sve_add,        INS_sve_add,        INS_sve_add,        INS_sve_add,        INS_sve_add,        INS_sve_add,        INS_sve_add,        INS_sve_fadd,       INS_sve_fadd},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           AddAcross,                                                        -1,      1,     {INS_sve_saddv,      INS_sve_uaddv,      INS_sve_saddv,      INS_sve_uaddv,      INS_sve_saddv,      INS_sve_uaddv,      INS_sve_uaddv,      INS_sve_uaddv,      INS_sve_faddv,      INS_sve_faddv},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           AddRotateComplex,                                                 -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcadd,      INS_sve_fcadd},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve,           AddSaturate,                                                      -1,      2,     {INS_sve_sqadd,      INS_sve_uqadd,      INS_sve_sqadd,      INS_sve_uqadd,      INS_sve_sqadd,      INS_sve_uqadd,      INS_sve_sqadd,      INS_sve_uqadd,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           AddSequentialAcross,                                              -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fadda,      INS_sve_fadda},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           And,                                                              -1,     -1,     {INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           AndAcross,                                                        -1,     -1,     {INS_sve_andv,       INS_sve_andv,       INS_sve_andv,       INS_sve_andv,       INS_sve_andv,       INS_sve_andv,       INS_sve_andv,       INS_sve_andv,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           BitwiseClear,                                                     -1,     -1,     {INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           BooleanNot,                                                       -1,     -1,     {INS_sve_cnot,       INS_sve_cnot,       INS_sve_cnot,       INS_sve_cnot,       INS_sve_cnot,       INS_sve_cnot,       INS_sve_cnot,       INS_sve_cnot,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Compact,                                                          -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_compact,    INS_sve_compact,    INS_sve_compact,    INS_sve_compact,    INS_sve_compact,    INS_sve_compact}, HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareEqual,                                                     -1,      -1,    {INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_cmpeq,      INS_sve_fcmeq,      INS_sve_fcmeq},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareGreaterThan,                                               -1,      -1,    {INS_sve_cmpgt,      INS_sve_cmphi,      INS_sve_cmpgt,      INS_sve_cmphi,      INS_sve_cmpgt,      INS_sve_cmphi,      INS_sve_cmpgt,      INS_sve_cmphi,      INS_sve_fcmgt,      INS_sve_fcmgt},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareGreaterThanOrEqual,                                        -1,      -1,    {INS_sve_cmpge,      INS_sve_cmphs,      INS_sve_cmpge,      INS_sve_cmphs,      INS_sve_cmpge,      INS_sve_cmphs,      INS_sve_cmpge,      INS_sve_cmphs,      INS_sve_fcmge,      INS_sve_fcmge},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareLessThan,                                                  -1,      -1,    {INS_sve_cmplt,      INS_sve_cmplo,      INS_sve_cmplt,      INS_sve_cmplo,      INS_sve_cmplt,      INS_sve_cmplo,      INS_sve_cmplt,      INS_sve_cmplo,      INS_sve_fcmlt,      INS_sve_fcmlt},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareLessThanOrEqual,                                           -1,      -1,    {INS_sve_cmple,      INS_sve_cmpls,      INS_sve_cmple,      INS_sve_cmpls,      INS_sve_cmple,      INS_sve_cmpls,      INS_sve_cmple,      INS_sve_cmpls,      INS_sve_fcmle,      INS_sve_fcmle},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareNotEqualTo,                                                -1,      -1,    {INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_fcmne,      INS_sve_fcmne},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CompareUnordered,                                                 -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcmuo,      INS_sve_fcmuo},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Compute16BitAddresses,                                            -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           Compute32BitAddresses,                                            -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           Compute64BitAddresses,                                            -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           Compute8BitAddresses,                                             -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_sve_adr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ConditionalExtractAfterLastActiveElement,                         -1,      3,     {INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ConditionalExtractAfterLastActiveElementAndReplicate,             -1,      3,     {INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           ConditionalExtractLastActiveElement,                              -1,      3,     {INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ConditionalExtractLastActiveElementAndReplicate,                  -1,      3,     {INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           ConditionalSelect,                                                -1,      3,     {INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           ConvertToDouble,                                                  -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_scvtf,      INS_sve_ucvtf,      INS_sve_scvtf,      INS_sve_ucvtf,      INS_sve_fcvt,       INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConvertToInt32,                                                   -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtzs,     INS_sve_fcvtzs},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConvertToInt64,                                                   -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtzs,     INS_sve_fcvtzs},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConvertToSingle,                                                  -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_scvtf,      INS_sve_ucvtf,      INS_sve_scvtf,      INS_sve_ucvtf,      INS_invalid,        INS_sve_fcvt},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConvertToUInt32,                                                  -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtzu,     INS_sve_fcvtzu},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConvertToUInt64,                                                  -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtzu,     INS_sve_fcvtzu},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Count16BitElements,                                                0,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_cnth,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed)
HARDWARE_INTRINSIC(Sve,           Count32BitElements,                                                0,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_cntw,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed)
HARDWARE_INTRINSIC(Sve,           Count64BitElements,                                                0,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_cntd,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed)
HARDWARE_INTRINSIC(Sve,           Count8BitElements,                                                 0,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_cntb,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed)
HARDWARE_INTRINSIC(Sve,           CreateBreakAfterMask,                                             -1,      2,     {INS_sve_brka,       INS_sve_brka,       INS_sve_brka,       INS_sve_brka,       INS_sve_brka,       INS_sve_brka,       INS_sve_brka,       INS_sve_brka,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           CreateBreakAfterPropagateMask,                                    -1,      3,     {INS_sve_brkpa,      INS_sve_brkpa,      INS_sve_brkpa,      INS_sve_brkpa,      INS_sve_brkpa,      INS_sve_brkpa,      INS_sve_brkpa,      INS_sve_brkpa,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CreateBreakBeforeMask,                                            -1,      2,     {INS_sve_brkb,       INS_sve_brkb,       INS_sve_brkb,       INS_sve_brkb,       INS_sve_brkb,       INS_sve_brkb,       INS_sve_brkb,       INS_sve_brkb,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           CreateBreakBeforePropagateMask,                                   -1,      3,     {INS_sve_brkpb,      INS_sve_brkpb,      INS_sve_brkpb,      INS_sve_brkpb,      INS_sve_brkpb,      INS_sve_brkpb,      INS_sve_brkpb,      INS_sve_brkpb,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CreateBreakPropagateMask,                                         -1,      -1,    {INS_sve_brkn,       INS_sve_brkn,       INS_sve_brkn,       INS_sve_brkn,       INS_sve_brkn,       INS_sve_brkn,       INS_sve_brkn,       INS_sve_brkn,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskByte,                                              -1,      0,     {INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskDouble,                                            -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskInt16,                                             -1,      0,     {INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskInt32,                                             -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskInt64,                                             -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskSByte,                                             -1,      0,     {INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskSingle,                                            -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskUInt16,                                            -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskUInt32,                                            -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateFalseMaskUInt64,                                            -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pfalse,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateMaskForFirstActiveElement,                                  -1,      2,     {INS_sve_pfirst,     INS_sve_pfirst,     INS_sve_pfirst,     INS_sve_pfirst,     INS_sve_pfirst,     INS_sve_pfirst,     INS_sve_pfirst,     INS_sve_pfirst,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           CreateMaskForNextActiveElement,                                   -1,      2,     {INS_invalid,        INS_sve_pnext,      INS_invalid,        INS_sve_pnext,      INS_invalid,        INS_sve_pnext,      INS_invalid,        INS_sve_pnext,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskByte,                                               -1,      1,     {INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskDouble,                                             -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskInt16,                                              -1,      1,     {INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskInt32,                                              -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskInt64,                                              -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskSByte,                                              -1,      1,     {INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskSingle,                                             -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskUInt16,                                             -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskUInt32,                                             -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateTrueMaskUInt64,                                             -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ptrue,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanMask16Bit,                                     -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_whilelt,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanMask32Bit,                                     -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_whilelt,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanMask64Bit,                                     -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_whilelt,    INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanMask8Bit,                                      -1,      2,     {INS_invalid,        INS_sve_whilelt,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanOrEqualMask16Bit,                              -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_whilele,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanOrEqualMask32Bit,                              -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_whilele,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanOrEqualMask64Bit,                              -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_whilele,    INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           CreateWhileLessThanOrEqualMask8Bit,                               -1,      2,     {INS_invalid,        INS_sve_whilele,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           Divide,                                                           -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sdiv,       INS_sve_udiv,       INS_sve_sdiv,       INS_sve_udiv,       INS_sve_fdiv,       INS_sve_fdiv},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           DotProduct,                                                       -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sdot,       INS_sve_udot,       INS_sve_sdot,       INS_sve_udot,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           DotProductBySelectedScalar,                                       -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sdot,       INS_sve_udot,       INS_sve_sdot,       INS_sve_udot,       INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve,           DuplicateSelectedScalarToVector,                                  -1,      2,     {INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup,        INS_sve_dup},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve,           ExtractAfterLastActiveElement,                                    -1,      2,     {INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ExtractAfterLastActiveElementScalar,                               0,      2,     {INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta,      INS_sve_lasta},   HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           ExtractLastActiveElement,                                         -1,      2,     {INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ExtractLastActiveElementScalar,                                    0,      2,     {INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb,      INS_sve_lastb},   HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           ExtractVector,                                                    -1,      3,     {INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext,        INS_sve_ext},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           FloatingPointExponentialAccelerator,                              -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fexpa,      INS_invalid,        INS_sve_fexpa,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           FusedMultiplyAdd,                                                 -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmla,       INS_sve_fmla},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           FusedMultiplyAddBySelectedScalar,                                 -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmla,       INS_sve_fmla},    HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve,           FusedMultiplyAddNegated,                                          -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fnmla,      INS_sve_fnmla},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           FusedMultiplySubtract,                                            -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmls,       INS_sve_fmls},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           FusedMultiplySubtractBySelectedScalar,                            -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmls,       INS_sve_fmls},    HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve,           FusedMultiplySubtractNegated,                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fnmls,      INS_sve_fnmls},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           GatherPrefetch16Bit,                                              -1,     -1,     {INS_invalid,        INS_invalid,        INS_sve_prfh,       INS_sve_prfh,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherPrefetch32Bit,                                              -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfw,       INS_sve_prfw,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherPrefetch64Bit,                                              -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfd,       INS_sve_prfd,       INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherPrefetch8Bit,                                               -1,     -1,     {INS_sve_prfb,       INS_sve_prfb,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVector,                                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1d,       INS_sve_ld1d,       INS_sve_ld1w,       INS_sve_ld1d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorByteZeroExtend,                                       -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_sve_ld1b,       INS_sve_ld1b,       INS_sve_ld1b,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorByteZeroExtendFirstFaulting,                          -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1b,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorFirstFaulting,                                        -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1d,     INS_sve_ldff1d,     INS_sve_ldff1w,     INS_sve_ldff1d},  HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt16SignExtend,                                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_sve_ld1sh,      INS_sve_ld1sh,      INS_sve_ld1sh,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt16SignExtendFirstFaulting,                         -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt16WithByteOffsetsSignExtend,                       -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_sve_ld1sh,      INS_sve_ld1sh,      INS_sve_ld1sh,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt16WithByteOffsetsSignExtendFirstFaulting,          -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt32SignExtend,                                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sw,      INS_sve_ld1sw,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt32SignExtendFirstFaulting,                         -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sw,    INS_sve_ldff1sw,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt32WithByteOffsetsSignExtend,                       -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sw,      INS_sve_ld1sw,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorInt32WithByteOffsetsSignExtendFirstFaulting,          -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sw,    INS_sve_ldff1sw,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorSByteSignExtend,                                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_sve_ld1sb,      INS_sve_ld1sb,      INS_sve_ld1sb,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorSByteSignExtendFirstFaulting,                         -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt16WithByteOffsetsZeroExtend,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1h,       INS_sve_ld1h,       INS_sve_ld1h,       INS_sve_ld1h,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt16WithByteOffsetsZeroExtendFirstFaulting,         -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt16ZeroExtend,                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1h,       INS_sve_ld1h,       INS_sve_ld1h,       INS_sve_ld1h,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt16ZeroExtendFirstFaulting,                        -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt32WithByteOffsetsZeroExtend,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1w,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt32WithByteOffsetsZeroExtendFirstFaulting,         -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1w,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt32ZeroExtend,                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1w,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GatherVectorUInt32ZeroExtendFirstFaulting,                        -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1w,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorWithByteOffsetFirstFaulting,                          -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1d,     INS_sve_ldff1d,     INS_sve_ldff1w,     INS_sve_ldff1d},  HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GatherVectorWithByteOffsets,                                      -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1d,       INS_sve_ld1d,       INS_sve_ld1w,       INS_sve_ld1d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GetActiveElementCount,                                            -1,      2,     {INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp,       INS_sve_cntp},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation)
HARDWARE_INTRINSIC(Sve,           GetFfrByte,                                                       -1,      0,     {INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrInt16,                                                      -1,      0,     {INS_invalid,        INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrInt32,                                                      -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrInt64,                                                      -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrSByte,                                                      -1,      0,     {INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrUInt16,                                                     -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrUInt32,                                                     -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           GetFfrUInt64,                                                     -1,      0,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_rdffr,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           InsertIntoShiftedVector,                                          -1,      2,     {INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr,       INS_sve_insr},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           LeadingSignCount,                                                 -1,     -1,     {INS_sve_cls,        INS_invalid,        INS_sve_cls,        INS_invalid,        INS_sve_cls,        INS_invalid,        INS_sve_cls,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LeadingZeroCount,                                                 -1,     -1,     {INS_sve_clz,        INS_sve_clz,        INS_sve_clz,        INS_sve_clz,        INS_sve_clz,        INS_sve_clz,        INS_sve_clz,        INS_sve_clz,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Load2xVectorAndUnzip,                                             -1,      2,     {INS_sve_ld2b,       INS_sve_ld2b,       INS_sve_ld2h,       INS_sve_ld2h,       INS_sve_ld2w,       INS_sve_ld2w,       INS_sve_ld2d,       INS_sve_ld2d,       INS_sve_ld2w,       INS_sve_ld2d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           Load3xVectorAndUnzip,                                             -1,      2,     {INS_sve_ld3b,       INS_sve_ld3b,       INS_sve_ld3h,       INS_sve_ld3h,       INS_sve_ld3w,       INS_sve_ld3w,       INS_sve_ld3d,       INS_sve_ld3d,       INS_sve_ld3w,       INS_sve_ld3d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           Load4xVectorAndUnzip,                                             -1,      2,     {INS_sve_ld4b,       INS_sve_ld4b,       INS_sve_ld4h,       INS_sve_ld4h,       INS_sve_ld4w,       INS_sve_ld4w,       INS_sve_ld4d,       INS_sve_ld4d,       INS_sve_ld4w,       INS_sve_ld4d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_MultiReg|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           LoadVector,                                                       -1,      2,     {INS_sve_ld1b,       INS_sve_ld1b,       INS_sve_ld1h,       INS_sve_ld1h,       INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1d,       INS_sve_ld1d,       INS_sve_ld1w,       INS_sve_ld1d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVector128AndReplicateToVector,                                -1,      2,     {INS_sve_ld1rqb,     INS_sve_ld1rqb,     INS_sve_ld1rqh,     INS_sve_ld1rqh,     INS_sve_ld1rqw,     INS_sve_ld1rqw,     INS_sve_ld1rqd,     INS_sve_ld1rqd,     INS_sve_ld1rqw,     INS_sve_ld1rqd},  HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteNonFaultingZeroExtendToInt16,                       -1,     -1,     {INS_invalid,        INS_invalid,        INS_sve_ldnf1b,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteNonFaultingZeroExtendToInt32,                       -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1b,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteNonFaultingZeroExtendToInt64,                       -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1b,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteNonFaultingZeroExtendToUInt16,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1b,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteNonFaultingZeroExtendToUInt32,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1b,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteNonFaultingZeroExtendToUInt64,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1b,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendFirstFaulting,                            -1,      -1,     {INS_invalid,        INS_invalid,        INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1b,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendToInt16,                                  -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendToInt32,                                  -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendToInt64,                                  -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendToUInt16,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendToUInt32,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorByteZeroExtendToUInt64,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorFirstFaulting,                                          -1,      -1,    {INS_sve_ldff1b,     INS_sve_ldff1b,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1w,     INS_sve_ldff1w,     INS_sve_ldff1d,     INS_sve_ldff1d,     INS_sve_ldff1w,     INS_sve_ldff1d},  HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16NonFaultingSignExtendToInt32,                      -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sh,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16NonFaultingSignExtendToInt64,                      -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sh,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16NonFaultingSignExtendToUInt32,                     -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sh,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16NonFaultingSignExtendToUInt64,                     -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sh,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16SignExtendFirstFaulting,                           -1,      -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_sve_ldff1sh,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16SignExtendToInt32,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16SignExtendToInt64,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16SignExtendToUInt32,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt16SignExtendToUInt64,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt32NonFaultingSignExtendToInt64,                      -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sw,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt32NonFaultingSignExtendToUInt64,                     -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sw,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt32SignExtendFirstFaulting,                           -1,      -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1sw,    INS_sve_ldff1sw,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt32SignExtendToInt64,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sw,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorInt32SignExtendToUInt64,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sw,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorNonFaulting,                                            -1,     -1,     {INS_sve_ldnf1b,     INS_sve_ldnf1b,     INS_sve_ldnf1h,     INS_sve_ldnf1h,     INS_sve_ldnf1w,     INS_sve_ldnf1w,     INS_sve_ldnf1d,     INS_sve_ldnf1d,     INS_sve_ldnf1w,     INS_sve_ldnf1d},  HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorNonTemporal,                                            -1,      2,     {INS_sve_ldnt1b,     INS_sve_ldnt1b,     INS_sve_ldnt1h,     INS_sve_ldnt1h,     INS_sve_ldnt1w,     INS_sve_ldnt1w,     INS_sve_ldnt1d,     INS_sve_ldnt1d,     INS_sve_ldnt1w,     INS_sve_ldnt1d},  HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteNonFaultingSignExtendToInt16,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_sve_ldnf1sb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteNonFaultingSignExtendToInt32,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteNonFaultingSignExtendToInt64,                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sb,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteNonFaultingSignExtendToUInt16,                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteNonFaultingSignExtendToUInt32,                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteNonFaultingSignExtendToUInt64,                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1sb,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendFirstFaulting,                           -1,     -1,      {INS_invalid,        INS_invalid,        INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_sve_ldff1sb,    INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendToInt16,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendToInt32,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendToInt64,                                 -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendToUInt16,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendToUInt32,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorSByteSignExtendToUInt64,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sb,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16NonFaultingZeroExtendToInt32,                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1h,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16NonFaultingZeroExtendToInt64,                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1h,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16NonFaultingZeroExtendToUInt32,                    -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1h,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16NonFaultingZeroExtendToUInt64,                    -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1h,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16ZeroExtendFirstFaulting,                          -1,      -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_sve_ldff1h,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16ZeroExtendToInt32,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1h,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16ZeroExtendToInt64,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1h,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16ZeroExtendToUInt32,                               -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1h,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt16ZeroExtendToUInt64,                               -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1h,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt32NonFaultingZeroExtendToInt64,                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1w,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt32NonFaultingZeroExtendToUInt64,                    -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldnf1w,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt32ZeroExtendFirstFaulting,                          -1,      -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ldff1w,     INS_sve_ldff1w,     INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_SpecialSideEffectMask)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt32ZeroExtendToInt64,                                -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           LoadVectorUInt32ZeroExtendToUInt64,                               -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ZeroingMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Max,                                                              -1,      -1,    {INS_sve_smax,       INS_sve_umax,       INS_sve_smax,       INS_sve_umax,       INS_sve_smax,       INS_sve_umax,       INS_sve_smax,       INS_sve_umax,       INS_sve_fmax,       INS_sve_fmax},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           MaxAcross,                                                        -1,      -1,    {INS_sve_smaxv,      INS_sve_umaxv,      INS_sve_smaxv,      INS_sve_umaxv,      INS_sve_smaxv,      INS_sve_umaxv,      INS_sve_smaxv,      INS_sve_umaxv,      INS_sve_fmaxv,      INS_sve_fmaxv},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           MaxNumber,                                                        -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmaxnm,     INS_sve_fmaxnm},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           MaxNumberAcross,                                                  -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmaxnmv,    INS_sve_fmaxnmv}, HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           Min,                                                              -1,      -1,    {INS_sve_smin,       INS_sve_umin,       INS_sve_smin,       INS_sve_umin,       INS_sve_smin,       INS_sve_umin,       INS_sve_smin,       INS_sve_umin,       INS_sve_fmin,       INS_sve_fmin},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           MinAcross,                                                        -1,      -1,    {INS_sve_sminv,      INS_sve_uminv,      INS_sve_sminv,      INS_sve_uminv,      INS_sve_sminv,      INS_sve_uminv,      INS_sve_sminv,      INS_sve_uminv,      INS_sve_fminv,      INS_sve_fminv},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           MinNumber,                                                        -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fminnm,     INS_sve_fminnm},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           MinNumberAcross,                                                  -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fminnmv,    INS_sve_fminnmv}, HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           Multiply,                                                         -1,      2,     {INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_fmul,       INS_sve_fmul},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           MultiplyAdd,                                                      -1,     -1,     {INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           MultiplyAddRotateComplex,                                         -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcmla,      INS_sve_fcmla},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve,           MultiplyAddRotateComplexBySelectedScalar,                         -1,      5,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcmla,      INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve,           MultiplyBySelectedScalar,                                         -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmul,       INS_sve_fmul},    HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve,           MultiplyExtended,                                                 -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmulx,      INS_sve_fmulx},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           MultiplySubtract,                                                 -1,     -1,     {INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           Negate,                                                           -1,     -1,     {INS_sve_neg,        INS_invalid,        INS_sve_neg,        INS_invalid,        INS_sve_neg,        INS_invalid,        INS_sve_neg,        INS_invalid,        INS_sve_fneg,       INS_sve_fneg},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Not,                                                              -1,      -1,    {INS_sve_not,        INS_sve_not,        INS_sve_not,        INS_sve_not,        INS_sve_not,        INS_sve_not,        INS_sve_not,        INS_sve_not,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Or,                                                               -1,     -1,     {INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           OrAcross,                                                         -1,     -1,     {INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           PopCount,                                                         -1,     -1,     {INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Prefetch16Bit,                                                    -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfh,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           Prefetch32Bit,                                                    -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfw,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           Prefetch64Bit,                                                    -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfd,       INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           Prefetch8Bit,                                                    -1,      3,     {INS_invalid,        INS_sve_prfb,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_SpecialSideEffect_Other)
HARDWARE_INTRINSIC(Sve,           ReciprocalEstimate,                                               -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frecpe,     INS_sve_frecpe},  HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           ReciprocalExponent,                                               -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frecpx,     INS_sve_frecpx},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ReciprocalSqrtEstimate,                                           -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frsqrte,    INS_sve_frsqrte}, HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           ReciprocalSqrtStep,                                               -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frsqrts,    INS_sve_frsqrts}, HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           ReciprocalStep,                                                   -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frecps,     INS_sve_frecps},  HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           ReverseBits,                                                      -1,     -1,     {INS_sve_rbit,       INS_sve_rbit,       INS_sve_rbit,       INS_sve_rbit,       INS_sve_rbit,       INS_sve_rbit,       INS_sve_rbit,       INS_sve_rbit,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ReverseElement,                                                   -1,      1,     {INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           ReverseElement16,                                                 -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_revh,       INS_sve_revh,       INS_sve_revh,       INS_sve_revh,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ReverseElement32,                                                 -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_revw,       INS_sve_revw,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ReverseElement8,                                                  -1,     -1,     {INS_invalid,        INS_invalid,        INS_sve_revb,       INS_sve_revb,       INS_sve_revb,       INS_sve_revb,       INS_sve_revb,       INS_sve_revb,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           RoundAwayFromZero,                                                -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frinta,     INS_sve_frinta},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           RoundToNearest,                                                   -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frintn,     INS_sve_frintn},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           RoundToNegativeInfinity,                                          -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frintm,     INS_sve_frintm},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           RoundToPositiveInfinity,                                          -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frintp,     INS_sve_frintp},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           RoundToZero,                                                      -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frintz,     INS_sve_frintz},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy16BitElementCount,                           -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdech,     INS_sve_uqdech,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy32BitElementCount,                           -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdecw,     INS_sve_uqdecw,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy64BitElementCount,                           -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdecd,     INS_sve_uqdecd,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy8BitElementCount,                             0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdecb,     INS_sve_uqdecb,     INS_sve_sqdecb,     INS_sve_uqdecb,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementByActiveElementCount,                          -1,      2,     {INS_invalid,        INS_sve_sqdecp,     INS_sve_sqdecp,     INS_sve_sqdecp,     INS_sve_sqdecp,     INS_sve_sqdecp,     INS_sve_sqdecp,     INS_sve_sqdecp,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy16BitElementCount,                           -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqinch,     INS_sve_uqinch,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy32BitElementCount,                           -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqincw,     INS_sve_uqincw,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy64BitElementCount,                           -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqincd,     INS_sve_uqincd,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_HasScalarInputVariant|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy8BitElementCount,                             0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqincb,     INS_sve_uqincb,     INS_sve_sqincb,     INS_sve_uqincb,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementByActiveElementCount,                          -1,      2,     {INS_invalid,        INS_sve_sqincp,     INS_sve_sqincp,     INS_sve_sqincp,     INS_sve_sqincp,     INS_sve_sqincp,     INS_sve_sqincp,     INS_sve_sqincp,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           Scale,                                                            -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fscale,     INS_sve_fscale},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           Scatter,                                                          -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1w,       INS_sve_st1w,       INS_sve_st1d,       INS_sve_st1d,       INS_sve_st1w,       INS_sve_st1d},    HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Scatter16BitNarrowing,                                            -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1h,       INS_sve_st1h,       INS_sve_st1h,       INS_sve_st1h,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Scatter16BitWithByteOffsetsNarrowing,                             -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1h,       INS_sve_st1h,       INS_sve_st1h,       INS_sve_st1h,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Scatter32BitNarrowing,                                            -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1w,       INS_sve_st1w,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Scatter32BitWithByteOffsetsNarrowing,                             -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1w,       INS_sve_st1w,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Scatter8BitNarrowing,                                             -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1b,       INS_sve_st1b,       INS_sve_st1b,       INS_sve_st1b,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Scatter8BitWithByteOffsetsNarrowing,                              -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1b,       INS_sve_st1b,       INS_sve_st1b,       INS_sve_st1b,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ScatterWithByteOffsets,                                           -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_st1w,       INS_sve_st1w,       INS_sve_st1d,       INS_sve_st1d,       INS_sve_st1w,       INS_sve_st1d},    HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           SetFfr,                                                           -1,      1,     {INS_sve_wrffr,      INS_sve_wrffr,      INS_sve_wrffr,      INS_sve_wrffr,      INS_sve_wrffr,      INS_sve_wrffr,      INS_sve_wrffr,      INS_sve_wrffr,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialSideEffect_Other|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ShiftLeftLogical,                                                 -1,      -1,    {INS_sve_lsl,        INS_sve_lsl,        INS_sve_lsl,        INS_sve_lsl,        INS_sve_lsl,        INS_sve_lsl,        INS_sve_lsl,        INS_sve_lsl,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           ShiftRightArithmetic,                                             -1,      -1,    {INS_sve_asr,        INS_invalid,        INS_sve_asr,        INS_invalid,        INS_sve_asr,        INS_invalid,        INS_sve_asr,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           ShiftRightArithmeticForDivide,                                    -1,      -1,    {INS_sve_asrd,       INS_invalid,        INS_sve_asrd,       INS_invalid,        INS_sve_asrd,       INS_invalid,        INS_sve_asrd,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve,           ShiftRightLogical,                                                -1,      -1,    {INS_invalid,        INS_sve_lsr,        INS_invalid,        INS_sve_lsr,        INS_invalid,        INS_sve_lsr,        INS_invalid,        INS_sve_lsr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SignExtend16,                                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sxth,       INS_invalid,        INS_sve_sxth,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           SignExtend32,                                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sxtw,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           SignExtend8,                                                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_sve_sxtb,       INS_invalid,        INS_sve_sxtb,       INS_invalid,        INS_sve_sxtb,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           SignExtendWideningLower,                                          -1,      1,     {INS_sve_sunpklo,    INS_invalid,        INS_sve_sunpklo,    INS_invalid,        INS_sve_sunpklo,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           SignExtendWideningUpper,                                          -1,      1,     {INS_sve_sunpkhi,    INS_invalid,        INS_sve_sunpkhi,    INS_invalid,        INS_sve_sunpkhi,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           Splice,                                                           -1,      3,     {INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice,     INS_sve_splice},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Sqrt,                                                             -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fsqrt,      INS_sve_fsqrt},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           StoreAndZip,                                                      -1,      3,     {INS_sve_st1b,       INS_sve_st1b,       INS_sve_st1h,       INS_sve_st1h,       INS_sve_st1w,       INS_sve_st1w,       INS_sve_st1d,       INS_sve_st1d,       INS_sve_st1w,       INS_sve_st1d},    HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           StoreNarrowing,                                                   -1,      3,     {INS_sve_st1b,       INS_sve_st1b,       INS_sve_st1h,       INS_sve_st1h,       INS_sve_st1w,       INS_sve_st1w,       INS_sve_st1d,       INS_sve_st1d,       INS_invalid,        INS_invalid},     HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           StoreNonTemporal,                                                 -1,      3,     {INS_sve_stnt1b,     INS_sve_stnt1b,     INS_sve_stnt1h,     INS_sve_stnt1h,     INS_sve_stnt1w,     INS_sve_stnt1w,     INS_sve_stnt1d,     INS_sve_stnt1d,     INS_sve_stnt1w,     INS_sve_stnt1d},  HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           Subtract,                                                         -1,      2,     {INS_sve_sub,        INS_sve_sub,        INS_sve_sub,        INS_sve_sub,        INS_sve_sub,        INS_sve_sub,        INS_sve_sub,        INS_sve_sub,        INS_sve_fsub,       INS_sve_fsub},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           SubtractSaturate,                                                 -1,      2,     {INS_sve_sqsub,      INS_sve_uqsub,      INS_sve_sqsub,      INS_sve_uqsub,      INS_sve_sqsub,      INS_sve_uqsub,      INS_sve_sqsub,      INS_sve_uqsub,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           TestAnyTrue,                                                      -1,      2,     {INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           TestFirstTrue,                                                    -1,      2,     {INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           TestLastTrue,                                                     -1,      2,     {INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_sve_ptest,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           TransposeEven,                                                    -1,      2,     {INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           TransposeOdd,                                                     -1,      2,     {INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           TrigonometricMultiplyAddCoefficient,                              -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ftmad,      INS_sve_ftmad},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           TrigonometricSelectCoefficient,                                   -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ftssel,     INS_sve_ftssel},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           TrigonometricStartingValue,                                       -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ftsmul,     INS_sve_ftsmul},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           UnzipEven,                                                        -1,      2,     {INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           UnzipOdd,                                                         -1,      2,     {INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           VectorTableLookup,                                                -1,      2,     {INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           Xor,                                                              -1,     -1,     {INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           XorAcross,                                                        -1,     -1,     {INS_sve_eorv,       INS_sve_eorv,       INS_sve_eorv,       INS_sve_eorv,       INS_sve_eorv,       INS_sve_eorv,       INS_sve_eorv,       INS_sve_eorv,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve,           ZeroExtend16,                                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_uxth,       INS_invalid,        INS_sve_uxth,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ZeroExtend32,                                                     -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_uxtw,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ZeroExtend8,                                                      -1,     -1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_uxtb,       INS_invalid,        INS_sve_uxtb,       INS_invalid,        INS_sve_uxtb,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ZeroExtendWideningLower,                                          -1,      1,     {INS_invalid,        INS_sve_uunpklo,    INS_invalid,        INS_sve_uunpklo,    INS_invalid,        INS_sve_uunpklo,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           ZeroExtendWideningUpper,                                          -1,      1,     {INS_invalid,        INS_sve_uunpkhi,    INS_invalid,        INS_sve_uunpkhi,    INS_invalid,        INS_sve_uunpkhi,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg)
HARDWARE_INTRINSIC(Sve,           ZipHigh,                                                          -1,      2,     {INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
HARDWARE_INTRINSIC(Sve,           ZipLow,                                                           -1,      2,     {INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasAllMaskVariant)
#define LAST_NI_Sve               NI_Sve_ZipLow

// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
//                 ISA            Function name                                              SIMD size  NumArg                                                                                                                 Instructions                                                                                        Category                           Flags
//                                                                                                                  {TYP_BYTE,           TYP_UBYTE,          TYP_SHORT,          TYP_USHORT,         TYP_INT,            TYP_UINT,           TYP_LONG,           TYP_ULONG,          TYP_FLOAT,          TYP_DOUBLE}
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
// SVE2 Intrinsics
#define FIRST_NI_Sve2             NI_Sve2_AbsSaturate
HARDWARE_INTRINSIC(Sve2,          AbsSaturate,                                                      -1,      -1,    {INS_sve_sqabs,      INS_invalid,        INS_sve_sqabs,      INS_invalid,        INS_sve_sqabs,      INS_invalid,        INS_sve_sqabs,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceAdd,                                            -1,      3,     {INS_sve_saba,       INS_sve_uaba,       INS_sve_saba,       INS_sve_uaba,       INS_sve_saba,       INS_sve_uaba,       INS_sve_saba,       INS_sve_uaba,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceWideningEven,                                   -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sabdlb,     INS_sve_uabdlb,     INS_sve_sabdlb,     INS_sve_uabdlb,     INS_sve_sabdlb,     INS_sve_uabdlb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceWideningLowerAndAddEven,                        -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sabalb,     INS_sve_uabalb,     INS_sve_sabalb,     INS_sve_uabalb,     INS_sve_sabalb,     INS_sve_uabalb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceWideningLowerAndAddOdd,                         -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sabalt,     INS_sve_uabalt,     INS_sve_sabalt,     INS_sve_uabalt,     INS_sve_sabalt,     INS_sve_uabalt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          AbsoluteDifferenceWideningOdd,                                    -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sabdlt,     INS_sve_uabdlt,     INS_sve_sabdlt,     INS_sve_uabdlt,     INS_sve_sabdlt,     INS_sve_uabdlt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          AddCarryWideningEven,                                             -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adclb,      INS_invalid,        INS_sve_adclb,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          AddCarryWideningOdd,                                              -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_adclt,      INS_invalid,        INS_sve_adclt,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          AddHighNarrowingEven,                                             -1,      2,     {INS_sve_addhnb,     INS_sve_addhnb,     INS_sve_addhnb,     INS_sve_addhnb,     INS_sve_addhnb,     INS_sve_addhnb,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          AddHighNarrowingOdd,                                              -1,      3,     {INS_sve_addhnt,     INS_sve_addhnt,     INS_sve_addhnt,     INS_sve_addhnt,     INS_sve_addhnt,     INS_sve_addhnt,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          AddPairwise,                                                      -1,      -1,    {INS_sve_addp,       INS_sve_addp,       INS_sve_addp,       INS_sve_addp,       INS_sve_addp,       INS_sve_addp,       INS_sve_addp,       INS_sve_addp,       INS_sve_faddp,      INS_sve_faddp},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve2,          AddPairwiseWideningAndAdd,                                        -1,      -1,    {INS_invalid,        INS_invalid,        INS_sve_sadalp,     INS_sve_uadalp,     INS_sve_sadalp,     INS_sve_uadalp,     INS_sve_sadalp,     INS_sve_uadalp,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          AddRotateComplex,                                                 -1,      3,     {INS_sve_cadd,       INS_sve_cadd,       INS_sve_cadd,       INS_sve_cadd,       INS_sve_cadd,       INS_sve_cadd,       INS_sve_cadd,       INS_sve_cadd,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          AddRoundedHighNarrowingEven,                                      -1,      2,     {INS_sve_raddhnb,    INS_sve_raddhnb,    INS_sve_raddhnb,    INS_sve_raddhnb,    INS_sve_raddhnb,    INS_sve_raddhnb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          AddRoundedHighNarrowingOdd,                                       -1,      3,     {INS_sve_raddhnt,    INS_sve_raddhnt,    INS_sve_raddhnt,    INS_sve_raddhnt,    INS_sve_raddhnt,    INS_sve_raddhnt,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          AddSaturate,                                                      -1,      -1,    {INS_sve_sqadd,      INS_sve_uqadd,      INS_sve_sqadd,      INS_sve_uqadd,      INS_sve_sqadd,      INS_sve_uqadd,      INS_sve_sqadd,      INS_sve_uqadd,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          AddSaturateRotateComplex,                                         -1,      3,     {INS_sve_sqcadd,     INS_invalid,        INS_sve_sqcadd,     INS_invalid,        INS_sve_sqcadd,     INS_invalid,        INS_sve_sqcadd,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          AddSaturateWithSignedAddend,                                      -1,      -1,    {INS_invalid,        INS_sve_usqadd,     INS_invalid,        INS_sve_usqadd,     INS_invalid,        INS_sve_usqadd,     INS_invalid,        INS_sve_usqadd,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          AddSaturateWithUnsignedAddend,                                    -1,      -1,    {INS_sve_suqadd,     INS_invalid,        INS_sve_suqadd,     INS_invalid,        INS_sve_suqadd,     INS_invalid,        INS_sve_suqadd,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          AddWideningEven,                                                  -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_saddwb,     INS_sve_uaddwb,     INS_sve_saddwb,     INS_sve_uaddwb,     INS_sve_saddwb,     INS_sve_uaddwb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          AddWideningEvenOdd,                                               -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_saddlbt,    INS_invalid,        INS_sve_saddlbt,    INS_invalid,        INS_sve_saddlbt,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          AddWideningOdd,                                                   -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_saddwt,     INS_sve_uaddwt,     INS_sve_saddwt,     INS_sve_uaddwt,     INS_sve_saddwt,     INS_sve_uaddwt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          BitwiseClearXor,                                                  -1,      3,     {INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_sve_bcax,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          BitwiseSelect,                                                    -1,      3,     {INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_sve_bsl,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          BitwiseSelectLeftInverted,                                        -1,      3,     {INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_sve_bsl1n,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          BitwiseSelectRightInverted,                                       -1,      3,     {INS_sve_bsl2n,      INS_sve_bsl2n,      INS_sve_bsl2n,      INS_sve_bsl2n,      INS_sve_bsl2n,      INS_sve_bsl2n,      INS_sve_bsl2n,      INS_sve_bsl2n,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ConvertToDoubleOdd,                                               -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtlt,     INS_sve_fcvtlt},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ConvertToSingleEvenRoundToOdd,                                    -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtx,      INS_sve_fcvtx},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ConvertToSingleOdd,                                               -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtnt,     INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ConvertToSingleOddRoundToOdd,                                     -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fcvtxnt,    INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          DotProductRotateComplex,                                          -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_cdot,       INS_invalid,        INS_sve_cdot,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          DotProductRotateComplexBySelectedIndex,                           -1,      5,     {INS_sve_cdot,       INS_invalid,        INS_sve_cdot,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg)
HARDWARE_INTRINSIC(Sve2,          FusedAddHalving,                                                  -1,      -1,    {INS_sve_shadd,      INS_sve_uhadd,      INS_sve_shadd,      INS_sve_uhadd,      INS_sve_shadd,      INS_sve_uhadd,      INS_sve_shadd,      INS_sve_uhadd,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          FusedAddRoundedHalving,                                           -1,      -1,    {INS_sve_srhadd,     INS_sve_urhadd,     INS_sve_srhadd,     INS_sve_urhadd,     INS_sve_srhadd,     INS_sve_urhadd,     INS_sve_srhadd,     INS_sve_urhadd,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          FusedSubtractHalving,                                             -1,      -1,    {INS_sve_shsub,      INS_sve_uhsub,      INS_sve_shsub,      INS_sve_uhsub,      INS_sve_shsub,      INS_sve_uhsub,      INS_sve_shsub,      INS_sve_uhsub,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          InterleavingXorEvenOdd,                                           -1,      3,     {INS_sve_eorbt,      INS_sve_eorbt,      INS_sve_eorbt,      INS_sve_eorbt,      INS_sve_eorbt,      INS_sve_eorbt,      INS_sve_eorbt,      INS_sve_eorbt,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          InterleavingXorOddEven,                                           -1,      3,     {INS_sve_eortb,      INS_sve_eortb,      INS_sve_eortb,      INS_sve_eortb,      INS_sve_eortb,      INS_sve_eortb,      INS_sve_eortb,      INS_sve_eortb,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          Log2,                                                             -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_flogb,      INS_invalid,        INS_sve_flogb,      INS_invalid,        INS_sve_flogb,      INS_sve_flogb},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          MaxNumberPairwise,                                                -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmaxnmp,    INS_sve_fmaxnmp}, HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MaxPairwise,                                                      -1,      -1,    {INS_sve_smaxp,      INS_sve_umaxp,      INS_sve_smaxp,      INS_sve_umaxp,      INS_sve_smaxp,      INS_sve_umaxp,      INS_sve_smaxp,      INS_sve_umaxp,      INS_sve_fmaxp,      INS_sve_fmaxp},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve2,          MinNumberPairwise,                                                -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fminnmp,    INS_sve_fminnmp}, HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MinPairwise,                                                      -1,      -1,    {INS_sve_sminp,      INS_sve_uminp,      INS_sve_sminp,      INS_sve_uminp,      INS_sve_sminp,      INS_sve_uminp,      INS_sve_sminp,      INS_sve_uminp,      INS_sve_fminp,      INS_sve_fminp},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_ReduceOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyAddBySelectedScalar,                                      -1,      4,     {INS_invalid,        INS_invalid,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_sve_mla,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyAddRotateComplex,                                         -1,      4,     {INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          MultiplyAddRotateComplexBySelectedScalar,                         -1,      5,     {INS_invalid,        INS_invalid,        INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_sve_cmla,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve2,          MultiplyAddRoundedDoublingSaturateHighRotateComplex,              -1,      4,     {INS_sve_sqrdcmlah,  INS_invalid,        INS_sve_sqrdcmlah,  INS_invalid,        INS_sve_sqrdcmlah,  INS_invalid,        INS_sve_sqrdcmlah,  INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          MultiplyAddRoundedDoublingSaturateHighRotateComplexBySelectedScalar, -1,      5,  {INS_invalid,        INS_invalid,        INS_sve_sqrdcmlah,  INS_invalid,        INS_sve_sqrdcmlah,  INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalar,                                         -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_sve_mul,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalarWideningEven,                             -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_smullb,     INS_sve_umullb,     INS_sve_smullb,     INS_sve_umullb,     INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalarWideningEvenAndAdd,                       -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_smlalb,     INS_sve_umlalb,     INS_sve_smlalb,     INS_sve_umlalb,     INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalarWideningEvenAndSubtract,                  -1,      4,     {INS_invalid,        INS_invalid,        INS_sve_smlslb,     INS_sve_umlslb,     INS_sve_smlslb,     INS_sve_umlslb,     INS_sve_smlslb,     INS_sve_umlslb,     INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalarWideningOdd,                              -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_smullt,     INS_sve_umullt,     INS_sve_smullt,     INS_sve_umullt,     INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalarWideningOddAndAdd,                        -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_smlalt,     INS_sve_umlalt,     INS_sve_smlalt,     INS_sve_umlalt,     INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyBySelectedScalarWideningOddAndSubtract,                   -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_smlslt,     INS_sve_umlslt,     INS_sve_smlslt,     INS_sve_umlslt,     INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingBySelectedScalarSaturateHigh,                     -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmulh,    INS_invalid,        INS_sve_sqdmulh,    INS_invalid,        INS_sve_sqdmulh,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingSaturateHigh,                                     -1,      2,     {INS_sve_sqdmulh,    INS_invalid,        INS_sve_sqdmulh,    INS_invalid,        INS_sve_sqdmulh,    INS_invalid,        INS_sve_sqdmulh,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningAndAddSaturateEven,                       -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmlalb,   INS_invalid,        INS_sve_sqdmlalb,   INS_invalid,        INS_sve_sqdmlalb,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningAndAddSaturateEvenOdd,                    -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmlalbt,  INS_invalid,        INS_sve_sqdmlalbt,  INS_invalid,        INS_sve_sqdmlalbt,  INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningAndAddSaturateOdd,                        -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmlalt,   INS_invalid,        INS_sve_sqdmlalt,   INS_invalid,        INS_sve_sqdmlalt,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningAndSubtractSaturateEven,                  -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmlslb,   INS_invalid,        INS_sve_sqdmlslb,   INS_invalid,        INS_sve_sqdmlslb,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningAndSubtractSaturateEvenOdd,               -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmlslbt,  INS_invalid,        INS_sve_sqdmlslbt,  INS_invalid,        INS_sve_sqdmlslbt,  INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningAndSubtractSaturateOdd,                   -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqdmlslt,   INS_invalid,        INS_sve_sqdmlslt,   INS_invalid,        INS_sve_sqdmlslt,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven,       -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdmlalb,   INS_invalid,        INS_sve_sqdmlalb,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd,        -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdmlalt,   INS_invalid,        INS_sve_sqdmlalt,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven,  -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdmlslb,   INS_invalid,        INS_sve_sqdmlslb,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd,   -1,      4,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdmlslt,   INS_invalid,        INS_sve_sqdmlslt,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningSaturateEven,                             -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sqdmullb,   INS_invalid,        INS_sve_sqdmullb,   INS_invalid,        INS_sve_sqdmullb,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningSaturateEvenBySelectedScalar,             -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdmullb,   INS_invalid,        INS_sve_sqdmullb,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningSaturateOdd,                              -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sqdmullt,   INS_invalid,        INS_sve_sqdmullt,   INS_invalid,        INS_sve_sqdmullt,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          MultiplyDoublingWideningSaturateOddBySelectedScalar,              -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdmullt,   INS_invalid,        INS_sve_sqdmullt,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyRoundedDoublingBySelectedScalarSaturateHigh,              -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_sqrdmulh,   INS_invalid,        INS_sve_sqrdmulh,   INS_invalid,        INS_sve_sqrdmulh,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyRoundedDoublingSaturateAndAddHigh,                        -1,      3,     {INS_sve_sqrdmlah,   INS_invalid,        INS_sve_sqrdmlah,   INS_invalid,        INS_sve_sqrdmlah,   INS_invalid,        INS_sve_sqrdmlah,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyRoundedDoublingSaturateAndSubtractHigh,                   -1,      3,     {INS_sve_sqrdmlsh,   INS_invalid,        INS_sve_sqrdmlsh,   INS_invalid,        INS_sve_sqrdmlsh,   INS_invalid,        INS_sve_sqrdmlsh,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyRoundedDoublingSaturateBySelectedScalarAndAddHigh,        -1,      4,     {INS_invalid,        INS_invalid,        INS_sve_sqrdmlah,   INS_invalid,        INS_sve_sqrdmlah,   INS_invalid,        INS_sve_sqrdmlah,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyRoundedDoublingSaturateBySelectedScalarAndSubtractHigh,   -1,      4,     {INS_invalid,        INS_invalid,        INS_sve_sqrdmlsh,   INS_invalid,        INS_sve_sqrdmlsh,   INS_invalid,        INS_sve_sqrdmlsh,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyRoundedDoublingSaturateHigh,                              -1,      2,     {INS_sve_sqrdmulh,   INS_invalid,        INS_sve_sqrdmulh,   INS_invalid,        INS_sve_sqrdmulh,   INS_invalid,        INS_sve_sqrdmulh,   INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          MultiplySubtractBySelectedScalar,                                 -1,      4,     {INS_invalid,        INS_invalid,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_sve_mls,        INS_invalid,        INS_invalid},     HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2,          MultiplyWideningEven,                                             -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_smullb,     INS_sve_umullb,     INS_sve_smullb,     INS_sve_umullb,     INS_sve_smullb,     INS_sve_umullb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          MultiplyWideningEvenAndAdd,                                       -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_smlalb,     INS_sve_umlalb,     INS_sve_smlalb,     INS_sve_umlalb,     INS_sve_smlalb,     INS_sve_umlalb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyWideningEvenAndSubtract,                                  -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_smlslb,     INS_sve_umlslb,     INS_sve_smlslb,     INS_sve_umlslb,     INS_sve_smlslb,     INS_sve_umlslb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyWideningOdd,                                              -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_smullt,     INS_sve_umullt,     INS_sve_smullt,     INS_sve_umullt,     INS_sve_smullt,     INS_sve_umullt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          MultiplyWideningOddAndAdd,                                        -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_smlalt,     INS_sve_umlalt,     INS_sve_smlalt,     INS_sve_umlalt,     INS_sve_smlalt,     INS_sve_umlalt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          MultiplyWideningOddAndSubtract,                                   -1,      3,     {INS_invalid,        INS_invalid,        INS_sve_smlslt,     INS_sve_umlslt,     INS_sve_smlslt,     INS_sve_umlslt,     INS_sve_smlslt,     INS_sve_umlslt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          NegateSaturate,                                                   -1,      -1,    {INS_sve_sqneg,      INS_invalid,        INS_sve_sqneg,      INS_invalid,        INS_sve_sqneg,      INS_invalid,        INS_sve_sqneg,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          PolynomialMultiply,                                               -1,      2,     {INS_sve_pmul,       INS_sve_pmul,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          PolynomialMultiplyWideningEven,                                   -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pmullb,     INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pmullb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          PolynomialMultiplyWideningOdd,                                    -1,      2,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pmullt,     INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_pmullt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          ReciprocalEstimate,                                               -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_urecpe,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ReciprocalSqrtEstimate,                                           -1,      -1,    {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ursqrte,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ShiftArithmeticRounded,                                           -1,      -1,    {INS_sve_srshl,      INS_invalid,        INS_sve_srshl,      INS_invalid,        INS_sve_srshl,      INS_invalid,        INS_sve_srshl,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ShiftArithmeticRoundedSaturate,                                   -1,      -1,    {INS_sve_sqrshl,     INS_invalid,        INS_sve_sqrshl,     INS_invalid,        INS_sve_sqrshl,     INS_invalid,        INS_sve_sqrshl,     INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ShiftArithmeticSaturate,                                          -1,      -1,    {INS_sve_sqshl,      INS_invalid,        INS_sve_sqshl,      INS_invalid,        INS_sve_sqshl,      INS_invalid,        INS_sve_sqshl,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2,          ShiftLeftAndInsert,                                               -1,      3,     {INS_sve_sli,        INS_sve_sli,        INS_sve_sli,        INS_sve_sli,        INS_sve_sli,        INS_sve_sli,        INS_sve_sli,        INS_sve_sli,        INS_invalid,        INS_invalid},     HW_Category_ShiftLeftByImmediate,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftLeftLogicalSaturate,                                         -1,      -1,    {INS_invalid,        INS_sve_uqshl,      INS_invalid,        INS_sve_uqshl,      INS_invalid,        INS_sve_uqshl,      INS_invalid,        INS_sve_uqshl,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftLeftLogicalSaturateUnsigned,                                 -1,      -1,    {INS_invalid,        INS_sve_sqshlu,     INS_invalid,        INS_sve_sqshlu,     INS_invalid,        INS_sve_sqshlu,     INS_invalid,        INS_sve_sqshlu,     INS_invalid,        INS_invalid},     HW_Category_ShiftLeftByImmediate,  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftLeftLogicalWideningEven,                                     -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sshllb,     INS_sve_ushllb,     INS_sve_sshllb,     INS_sve_ushllb,     INS_sve_sshllb,     INS_sve_ushllb,     INS_invalid,        INS_invalid},     HW_Category_ShiftLeftByImmediate,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftLeftLogicalWideningOdd,                                      -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_sshllt,     INS_sve_ushllt,     INS_sve_sshllt,     INS_sve_ushllt,     INS_sve_sshllt,     INS_sve_ushllt,     INS_invalid,        INS_invalid},     HW_Category_ShiftLeftByImmediate,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftLogicalRounded,                                              -1,      -1,    {INS_invalid,        INS_sve_urshl,      INS_invalid,        INS_sve_urshl,      INS_invalid,        INS_sve_urshl,      INS_invalid,        INS_sve_urshl,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftLogicalRoundedSaturate,                                      -1,      -1,    {INS_invalid,        INS_sve_uqrshl,     INS_invalid,        INS_sve_uqrshl,     INS_invalid,        INS_sve_uqrshl,     INS_invalid,        INS_sve_uqrshl,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightAndInsert,                                              -1,      3,     {INS_sve_sri,        INS_sve_sri,        INS_sve_sri,        INS_sve_sri,        INS_sve_sri,        INS_sve_sri,        INS_sve_sri,        INS_sve_sri,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticAdd,                                          -1,      3,     {INS_sve_ssra,       INS_invalid,        INS_sve_ssra,       INS_invalid,        INS_sve_ssra,       INS_invalid,        INS_sve_ssra,       INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticNarrowingSaturateEven,                        -1,      2,     {INS_sve_sqshrnb,    INS_invalid,        INS_sve_sqshrnb,    INS_invalid,        INS_sve_sqshrnb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticNarrowingSaturateOdd,                         -1,      3,     {INS_sve_sqshrnt,    INS_invalid,        INS_sve_sqshrnt,    INS_invalid,        INS_sve_sqshrnt,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticNarrowingSaturateUnsignedEven,                -1,      2,     {INS_invalid,        INS_sve_sqshrunb,   INS_invalid,        INS_sve_sqshrunb,   INS_invalid,        INS_sve_sqshrunb,   INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticNarrowingSaturateUnsignedOdd,                 -1,      3,     {INS_invalid,        INS_sve_sqshrunt,   INS_invalid,        INS_sve_sqshrunt,   INS_invalid,        INS_sve_sqshrunt,   INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticRounded,                                      -1,      -1,    {INS_sve_srshr,      INS_invalid,        INS_sve_srshr,      INS_invalid,        INS_sve_srshr,      INS_invalid,        INS_sve_srshr,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticRoundedAdd,                                   -1,      3,     {INS_sve_srsra,      INS_invalid,        INS_sve_srsra,      INS_invalid,        INS_sve_srsra,      INS_invalid,        INS_sve_srsra,      INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticRoundedNarrowingSaturateEven,                 -1,      2,     {INS_sve_sqrshrnb,   INS_invalid,        INS_sve_sqrshrnb,   INS_invalid,        INS_sve_sqrshrnb,   INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticRoundedNarrowingSaturateOdd,                  -1,      3,     {INS_sve_sqrshrnt,   INS_invalid,        INS_sve_sqrshrnt,   INS_invalid,        INS_sve_sqrshrnt,   INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticRoundedNarrowingSaturateUnsignedEven,         -1,      2,     {INS_invalid,        INS_sve_sqrshrunb,  INS_invalid,        INS_sve_sqrshrunb,  INS_invalid,        INS_sve_sqrshrunb,  INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightArithmeticRoundedNarrowingSaturateUnsignedOdd,          -1,      3,     {INS_invalid,        INS_sve_sqrshrunt,  INS_invalid,        INS_sve_sqrshrunt,  INS_invalid,        INS_sve_sqrshrunt,  INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalAdd,                                             -1,      3,     {INS_invalid,        INS_sve_usra,       INS_invalid,        INS_sve_usra,       INS_invalid,        INS_sve_usra,       INS_invalid,        INS_sve_usra,       INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalNarrowingEven,                                   -1,      2,     {INS_sve_shrnb,      INS_sve_shrnb,      INS_sve_shrnb,      INS_sve_shrnb,      INS_sve_shrnb,      INS_sve_shrnb,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalNarrowingOdd,                                    -1,      3,     {INS_sve_shrnt,      INS_sve_shrnt,      INS_sve_shrnt,      INS_sve_shrnt,      INS_sve_shrnt,      INS_sve_shrnt,      INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalRounded,                                         -1,      -1,    {INS_invalid,        INS_sve_urshr,      INS_invalid,        INS_sve_urshr,      INS_invalid,        INS_sve_urshr,      INS_invalid,        INS_sve_urshr,      INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalRoundedAdd,                                      -1,      3,     {INS_invalid,        INS_sve_ursra,      INS_invalid,        INS_sve_ursra,      INS_invalid,        INS_sve_ursra,      INS_invalid,        INS_sve_ursra,      INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalRoundedNarrowingEven,                            -1,      2,     {INS_sve_rshrnb,     INS_sve_rshrnb,     INS_sve_rshrnb,     INS_sve_rshrnb,     INS_sve_rshrnb,     INS_sve_rshrnb,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalRoundedNarrowingOdd,                             -1,      3,     {INS_sve_rshrnt,     INS_sve_rshrnt,     INS_sve_rshrnt,     INS_sve_rshrnt,     INS_sve_rshrnt,     INS_sve_rshrnt,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalRoundedNarrowingSaturateEven,                    -1,      2,     {INS_invalid,        INS_sve_uqrshrnb,   INS_invalid,        INS_sve_uqrshrnb,   INS_invalid,        INS_sve_uqrshrnb,   INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand)
HARDWARE_INTRINSIC(Sve2,          ShiftRightLogicalRoundedNarrowingSaturateOdd,                     -1,      3,     {INS_invalid,        INS_sve_uqrshrnt,   INS_invalid,        INS_sve_uqrshrnt,   INS_invalid,        INS_sve_uqrshrnt,   INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          SubtractBorrowWideningEven,                                       -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sbclb,      INS_invalid,        INS_sve_sbclb,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          SubtractBorrowWideningOdd,                                        -1,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sbclt,      INS_invalid,        INS_sve_sbclt,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          SubtractHighNarrowingEven,                                        -1,      2,     {INS_sve_subhnb,     INS_sve_subhnb,     INS_sve_subhnb,     INS_sve_subhnb,     INS_sve_subhnb,     INS_sve_subhnb,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          SubtractHighNarrowingOdd,                                         -1,      3,     {INS_sve_subhnt,     INS_sve_subhnt,     INS_sve_subhnt,     INS_sve_subhnt,     INS_sve_subhnt,     INS_sve_subhnt,     INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          SubtractRoundedHighNarrowingEven,                                 -1,      2,     {INS_sve_rsubhnb,    INS_sve_rsubhnb,    INS_sve_rsubhnb,    INS_sve_rsubhnb,    INS_sve_rsubhnb,    INS_sve_rsubhnb,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          SubtractRoundedHighNarrowingOdd,                                  -1,      3,     {INS_sve_rsubhnt,    INS_sve_rsubhnt,    INS_sve_rsubhnt,    INS_sve_rsubhnt,    INS_sve_rsubhnt,    INS_sve_rsubhnt,    INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          SubtractSaturate,                                                 -1,      -1,    {INS_sve_sqsub,      INS_sve_uqsub,      INS_sve_sqsub,      INS_sve_uqsub,      INS_sve_sqsub,      INS_sve_uqsub,      INS_sve_sqsub,      INS_sve_uqsub,      INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          SubtractWideningEven,                                             -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_ssubwb,     INS_sve_usubwb,     INS_sve_ssubwb,     INS_sve_usubwb,     INS_sve_ssubwb,     INS_sve_usubwb,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          SubtractWideningEvenOdd,                                          -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_ssublbt,    INS_invalid,        INS_sve_ssublbt,    INS_invalid,        INS_sve_ssublbt,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          SubtractWideningOdd,                                              -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_ssubwt,     INS_sve_usubwt,     INS_sve_ssubwt,     INS_sve_usubwt,     INS_sve_ssubwt,     INS_sve_usubwt,     INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          SubtractWideningOddEven,                                          -1,      2,     {INS_invalid,        INS_invalid,        INS_sve_ssubltb,    INS_invalid,        INS_sve_ssubltb,    INS_invalid,        INS_sve_ssubltb,    INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2,          VectorTableLookup,                                                -1,      2,     {INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl,        INS_sve_tbl},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_NeedsConsecutiveRegisters|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve2,          VectorTableLookupExtension,                                       -1,      3,     {INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx,        INS_sve_tbx},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          Xor,                                                              -1,      3,     {INS_sve_eor3,       INS_sve_eor3,       INS_sve_eor3,       INS_sve_eor3,       INS_sve_eor3,       INS_sve_eor3,       INS_sve_eor3,       INS_sve_eor3,       INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2,          XorRotateRight,                                                   -1,      3,     {INS_sve_xar,        INS_sve_xar,        INS_sve_xar,        INS_sve_xar,        INS_sve_xar,        INS_sve_xar,        INS_sve_xar,        INS_sve_xar,        INS_invalid,        INS_invalid},     HW_Category_ShiftRightByImmediate, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand)
#define LAST_NI_Sve2              NI_Sve2_XorRotateRight

// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
//                 ISA            Function name                                              SIMD size  NumArg                                                                                                                 Instructions                                                                                        Category                           Flags
//                                                                                                                  {TYP_BYTE,           TYP_UBYTE,          TYP_SHORT,          TYP_USHORT,         TYP_INT,            TYP_UINT,           TYP_LONG,           TYP_ULONG,          TYP_FLOAT,          TYP_DOUBLE}
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
// Special intrinsics that are generated during importing or lowering
#define SPECIAL_NI_Sve            NI_Sve_ConditionalExtractAfterLastActiveElementScalar
HARDWARE_INTRINSIC(Sve,           ConditionalExtractAfterLastActiveElementScalar,                    0,      3,     {INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta,     INS_sve_clasta},  HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConditionalExtractLastActiveElementScalar,                         0,      3,     {INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb,     INS_sve_clastb},  HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve,           ConvertMaskToVector,                                              -1,      1,     {INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov,        INS_sve_mov},     HW_Category_Helper,                HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve,           ConvertVectorToMask,                                              -1,      2,     {INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne,      INS_sve_cmpne},   HW_Category_Helper,                HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation)
// True mask only used inside a ConvertVectorToMask
HARDWARE_INTRINSIC(Sve,           ConversionTrueMask,                                               -1,      0,     {INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue,      INS_sve_ptrue},   HW_Category_Helper,           	  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
// Scalar variants of Saturating*By*BitElementCount. There is 8bit versions as the generic version is scalar only.
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy16BitElementCountScalar,                      0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdech,     INS_sve_uqdech,     INS_sve_sqdech,     INS_sve_uqdech,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy32BitElementCountScalar,                      0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdecw,     INS_sve_uqdecw,     INS_sve_sqdecw,     INS_sve_uqdecw,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingDecrementBy64BitElementCountScalar,                      0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqdecd,     INS_sve_uqdecd,     INS_sve_sqdecd,     INS_sve_uqdecd,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy16BitElementCountScalar,                      0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqinch,     INS_sve_uqinch,     INS_sve_sqinch,     INS_sve_uqinch,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy32BitElementCountScalar,                      0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqincw,     INS_sve_uqincw,     INS_sve_sqincw,     INS_sve_uqincw,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           SaturatingIncrementBy64BitElementCountScalar,                      0,      3,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_sqincd,     INS_sve_uqincd,     INS_sve_sqincd,     INS_sve_uqincd,     INS_invalid,        INS_invalid},     HW_Category_Scalar,                HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_SpecialCodeGen|HW_Flag_SpecialImport|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve,           StoreAndZipx2,                                                    -1,      3,     {INS_sve_st2b,       INS_sve_st2b,       INS_sve_st2h,       INS_sve_st2h,       INS_sve_st2w,       INS_sve_st2w,       INS_sve_st2d,       INS_sve_st2d,       INS_sve_st2w,       INS_sve_st2d},    HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters)
HARDWARE_INTRINSIC(Sve,           StoreAndZipx3,                                                    -1,      3,     {INS_sve_st3b,       INS_sve_st3b,       INS_sve_st3h,       INS_sve_st3h,       INS_sve_st3w,       INS_sve_st3w,       INS_sve_st3d,       INS_sve_st3d,       INS_sve_st3w,       INS_sve_st3d},    HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters)
HARDWARE_INTRINSIC(Sve,           StoreAndZipx4,                                                    -1,      3,     {INS_sve_st4b,       INS_sve_st4b,       INS_sve_st4h,       INS_sve_st4h,       INS_sve_st4w,       INS_sve_st4w,       INS_sve_st4d,       INS_sve_st4d,       INS_sve_st4w,       INS_sve_st4d},    HW_Category_MemoryStore,           HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_NeedsConsecutiveRegisters)
// Predicate variants of intrinsics, these are specialized for operating on TYP_MASK type values.
HARDWARE_INTRINSIC(Sve,           And_Predicates,                                                   -1,      2,     {INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_sve_and,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           BitwiseClear_Predicates,                                          -1,      2,     {INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_sve_bic,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           Or_Predicates,                                                    -1,      2,     {INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           Xor_Predicates,                                                   -1,      2,     {INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_sve_eor,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_EmbeddedMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ConditionalSelect_Predicates,                                     -1,      3,     {INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel,        INS_sve_sel},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen)
HARDWARE_INTRINSIC(Sve,           ZipHigh_Predicates,                                               -1,      2,     {INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2,       INS_sve_zip2},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           ZipLow_Predicates,                                                -1,      2,     {INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1,       INS_sve_zip1},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           UnzipEven_Predicates,                                             -1,      2,     {INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1,       INS_sve_uzp1},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           UnzipOdd_Predicates,                                              -1,      2,     {INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2,       INS_sve_uzp2},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           TransposeEven_Predicates,                                         -1,      2,     {INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1,       INS_sve_trn1},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           TransposeOdd_Predicates,                                          -1,      2,     {INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2,       INS_sve_trn2},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)
HARDWARE_INTRINSIC(Sve,           ReverseElement_Predicates,                                        -1,      1,     {INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev,        INS_sve_rev},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask)

#endif // FEATURE_HW_INTRINSIC

#undef HARDWARE_INTRINSIC

// clang-format on
